package fr.cephb.operonj.tools.cmdline.remapper;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;

import java.io.PrintStream;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;

import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamWriter;

import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;

import com.sleepycat.bind.tuple.IntegerBinding;
import com.sleepycat.bind.tuple.StringBinding;
import com.sleepycat.bind.tuple.TupleBinding;
import com.sleepycat.bind.tuple.TupleInput;
import com.sleepycat.bind.tuple.TupleOutput;
import com.sleepycat.je.DatabaseConfig;
import com.sleepycat.je.DatabaseEntry;
import com.sleepycat.je.DatabaseException;
import com.sleepycat.je.EnvironmentConfig;
import com.sleepycat.je.OperationStatus;
import com.sleepycat.je.Transaction;

//import fr.cephb.operonj.core.bio.Observed;
import fr.cephb.operonj.core.db.database.SetOfIntegersDB;
import fr.cephb.operonj.core.je.DefaultPrimaryDB;
import fr.cephb.operonj.core.je.PrimaryDB;
import fr.cephb.operonj.core.je.Walker;
import fr.cephb.operonj.tools.cmdline.AbstractApplication;
import fr.cephb.operonj.tools.cmdline.remapper.BowtieHit.CandidatePair;

/**
 * Build37Mapper
 *
 */
public class Build37Mapper
	extends AbstractApplication
	{
	/** pattern for a simple mutation */
	private final static  Pattern TAB=Pattern.compile("[\t]");
	private final static  Pattern COMMA=Pattern.compile("[,]");
	/** name of the bowtie database */
	private File bowtieDBDir=new File("/operon/lindenb/ucsc/37/bowtie");//TODO
	/** all the bowtie files */
	private Set<String> all_bowtite_files=new HashSet<String>();
	/** current progame */
	private String program=null;
	/** rsId to RsInfo */
	private PrimaryDB<Integer, RsInfo> rsToInfo=null;
	/** where is bowtie */
	private File bowtieHome=new File("/operon/lindenb/bowtie/bowtie-0.11.3/");//TODO relative path
	/** path to blat */
	private File blatPath=new File("/home/operon/bin/i686/blat");//TODO
	/** where are the FASTA sequences for blat */
	private File fastaDir=new File("/operon/lindenb/ucsc/37/fasta");//TODO
	
	/** current of rs to be handler */
	private SetOfIntegersDB currentRsList;
	private int flankingLength=60;
	//private final int buildDest=37;
	private boolean keep_only_uniq_match=false;
	
	/** map chrom to integer */
	private Map<String,Integer> chrom2int=new HashMap<String,Integer>(100);
	
	
	
	/**
	 * Information about a SNP
	 * @author pierre
	 *
	 */
	public static class RsInfo
		{
		/** rs#### */
		int rsId;
		/** sequence on 5' */
		String seq5;
		/** observed sequence */
		String observed;
		/** sequence on 3' */
		String seq3;
		/** ncbi handles */
		Set<String> ncbiHandles=new HashSet<String>();
		/** hit count on left size */
		int leftHitCount=0;
		/** hit count on right size */
		int rightHitCount=0;
		/** matchers on chromosome 37 */
		List<Match> positions37=new ArrayList<Match>();
		
		/**
		 * Match on build37
		 * @author pierre
		 *
		 */
		static class Match
			{
			//int buildId=-1;
			int chromId;
			int start;
			int end;
			byte strand;
			int bound_size=-1;
			String comment="";
			}
		
		public String getSeq5(int length)
			{
			length=Math.min(seq5.length(), length);
			return seq5.substring(seq5.length()-length,seq5.length());
			}
		public String getSeq3(int length)
			{
			assert(length>=0);
			length=Math.min(seq3.length(), length);
			return seq3.substring(0,length);
			}
		
		public boolean isMapped()
			{
			return !this.positions37.isEmpty();
			}
		
		public boolean isMappedOnce()
			{
			return this.positions37.size()==1;
			}
		
		@Override
		public String toString() {
			return seq5+"["+observed+"]"+seq3;
			}
		}
	
	private static class RsInfoBinding
	extends TupleBinding<RsInfo>
		{
		@Override
		public RsInfo entryToObject(TupleInput in)
			{
			RsInfo info= new RsInfo();
			info.rsId= in.readInt();
			info.seq5= in.readString();
			info.observed= in.readString();
			info.seq3= in.readString();
			info.leftHitCount= in.readInt();
			info.rightHitCount= in.readInt();
			int n= in.readInt();
			for(int i=0;i< n;++i)
				{
				info.ncbiHandles.add(in.readString());
				}
			n= in.readInt();
			for(int i=0;i< n;++i)
				{
				RsInfo.Match seq=new RsInfo.Match();
				
				seq.chromId=in.readInt();
				seq.start=in.readInt();
				seq.end=in.readInt();
				seq.strand=in.readByte();
				seq.bound_size=in.readInt();
				seq.comment=in.readString();
				
				info.positions37.add(seq);
				}
			return info;
			}
		@Override
		public void objectToEntry(RsInfo o, TupleOutput out)
			{
			out.writeInt(o.rsId);
			out.writeString(o.seq5);
			out.writeString(o.observed);
			out.writeString(o.seq3);
			out.writeInt(o.leftHitCount);
			out.writeInt(o.rightHitCount);
			out.writeInt(o.ncbiHandles.size());
			for(String s:o.ncbiHandles)
				{
				out.writeString(s);
				}
			out.writeInt(o.positions37.size());
			for(RsInfo.Match seq:o.positions37)
				{
				//out.writeInt(seq.buildId);
				out.writeInt(seq.chromId);
				out.writeInt(seq.start);
				out.writeInt(seq.end);
				out.writeByte(seq.strand);
				out.writeInt(seq.bound_size);
				out.writeString(seq.comment);
				}
			}
		}
	
	/**
	 * Read a DBSNP XML file
	 */
	private class DBSnpHandler
	extends DefaultHandler
		{
		//private static final String NS="http://www.ncbi.nlm.nih.gov/SNP/docsum";
		private StringBuilder builder=null;
		private Integer rsId=null;
		private String seq5=null;
		private String observed=null;
		private String seq3=null;
		private Set<String> handle= new HashSet<String>();
		private boolean inSequence=false;
		
		private Transaction txn;
		public DBSnpHandler(
				Transaction txn
				)
			{
			this.txn=txn;			
			clear();
			}
		
		private void clear()
			{
			rsId=null;
			seq5=null;
			observed=null;
			seq3=null;
			inSequence=false;
			handle.clear();
			}
		
		@Override
		public void startDocument() throws SAXException
			{
			clear();
			this.builder=null;
			}
		
		@Override
		public void startElement(
				String uri, String localName, String qName,
				Attributes attributes) throws SAXException
			{
			if(localName.equals("Rs"))
				{
				rsId= Integer.parseInt(attributes.getValue("rsId"));
				assert(rsId>0);
				}
			else if(localName.equals("Ss"))
				{
				String s= attributes.getValue("handle");
				if(s!=null) this.handle.add(s);
				}
			else if(localName.equals("Seq5")||
					localName.equals("Observed") ||
					localName.equals("Seq3"))
				{
				assert(rsId!=null);
				this.builder= new StringBuilder();
				}
			else if(localName.equals("Sequence") &&
					attributes.getValue("exemplarSs")!=null)
				{
				assert(rsId!=null);
				assert(seq5==null);
				inSequence=true;
				}
			}
		@Override
		public void endElement(String uri, String localName, String qName) 
				throws SAXException
			{
			if(localName.equals("Rs"))
				{
				assert(rsId!=null);
				assert(seq5!=null);
				assert(seq3!=null);
				assert(observed!=null);
				try
					{
					//make a new RsInfo
					RsInfo info = new RsInfo();
					info.rsId=this.rsId;
					info.seq5=seq5;
					info.observed=this.observed;
					info.seq3=seq3;
					info.ncbiHandles.addAll(this.handle);
					if(Build37Mapper.this.rsToInfo.put(this.txn,  rsId, info)!=OperationStatus.SUCCESS)
						{
						throw new RuntimeException("Cannot insert "+rsId);
						}
					}
				catch (DatabaseException err)
					{
					throw new SAXException(err);
					}
				clear();
				}
			else if(inSequence && localName.equals("Seq5") )
				{
				if(seq5!=null)
					{
					LOG.info(""+rsId+" "+this.builder);
					}
				assert(seq5==null);
				seq5=this.builder.toString();
				}
			else if(inSequence && localName.equals("Seq3"))
				{
				assert(seq3==null);
				seq3=this.builder.toString();
				}
			else if(inSequence && localName.equals("Observed"))
				{
				assert(observed==null);
				observed=this.builder.toString();
				
				}
			else if(localName.equals("Sequence"))
				{
				inSequence=false;
				}
			this.builder=null;
			}
		
		@Override
		public void characters(char[] ch, int start, int length)
				throws SAXException
			{
			if(this.builder!=null) this.builder.append(ch,start,length);
			}
		
	
		}

	
	/**
	 * simple task performing a mapping
	 * @author pierre
	 *
	 */
	static interface MappingTask
		{
		public void init(Transaction txn) throws  Exception;
		public void dispose() throws  Exception;
		public void map(Transaction txn) throws  Exception;
		public int getPoolSize();
		}
	
	/**
	 * Constructor
	 */
	private  Build37Mapper()
		{
		super.envCfg.setAllowCreate(true);
		super.envCfg.setReadOnly(false);
		super.envCfg.setTransactional(false);
		super.envCfg.setConfigParam(EnvironmentConfig.LOG_FILE_MAX,"200000000"/* 200 Mo */);
		
		
		chrom2int.put("11_gl000202_random",26);
		chrom2int.put("17_ctg5_hap1",27);
		chrom2int.put("17_gl000203_random",28);
		chrom2int.put("17_gl000204_random",29);
		chrom2int.put("17_gl000205_random",30);
		chrom2int.put("17_gl000206_random",31);
		chrom2int.put("18_gl000207_random",32);
		chrom2int.put("19_gl000208_random",33);
		chrom2int.put("19_gl000209_random",34);
		chrom2int.put("1_gl000191_random",35);
		chrom2int.put("1_gl000192_random",36);
		chrom2int.put("21_gl000210_random",37);
		chrom2int.put("4_ctg9_hap1",38);
		chrom2int.put("4_gl000193_random",39);
		chrom2int.put("4_gl000194_random",40);
		chrom2int.put("6_apd_hap1",41);
		chrom2int.put("6_cox_hap2",42);
		chrom2int.put("6_dbb_hap3",43);
		chrom2int.put("6_mann_hap4",44);
		chrom2int.put("6_mcf_hap5",45);
		chrom2int.put("6_qbl_hap6",46);
		chrom2int.put("6_ssto_hap7",47);
		chrom2int.put("7_gl000195_random",48);
		chrom2int.put("8_gl000196_random",49);
		chrom2int.put("8_gl000197_random",50);
		chrom2int.put("9_gl000198_random",51);
		chrom2int.put("9_gl000199_random",52);
		chrom2int.put("9_gl000200_random",53);
		chrom2int.put("9_gl000201_random",54);
		chrom2int.put("un_gl000211",55);
		chrom2int.put("un_gl000212",56);
		chrom2int.put("un_gl000213",57);
		chrom2int.put("un_gl000214",58);
		chrom2int.put("un_gl000215",59);
		chrom2int.put("un_gl000216",60);
		chrom2int.put("un_gl000217",61);
		chrom2int.put("un_gl000218",62);
		chrom2int.put("un_gl000219",63);
		chrom2int.put("un_gl000220",64);
		chrom2int.put("un_gl000221",65);
		chrom2int.put("un_gl000222",66);
		chrom2int.put("un_gl000223",67);
		chrom2int.put("un_gl000224",68);
		chrom2int.put("un_gl000225",69);
		chrom2int.put("un_gl000226",70);
		chrom2int.put("un_gl000227",71);
		chrom2int.put("un_gl000228",72);
		chrom2int.put("un_gl000229",73);
		chrom2int.put("un_gl000230",74);
		chrom2int.put("un_gl000231",75);
		chrom2int.put("un_gl000232",76);
		chrom2int.put("un_gl000233",77);
		chrom2int.put("un_gl000234",78);
		chrom2int.put("un_gl000235",79);
		chrom2int.put("un_gl000236",80);
		chrom2int.put("un_gl000237",81);
		chrom2int.put("un_gl000238",82);
		chrom2int.put("un_gl000239",83);
		chrom2int.put("un_gl000240",84);
		chrom2int.put("un_gl000241",85);
		chrom2int.put("un_gl000242",86);
		chrom2int.put("un_gl000243",87);
		chrom2int.put("un_gl000244",88);
		chrom2int.put("un_gl000245",89);
		chrom2int.put("un_gl000246",90);
		chrom2int.put("un_gl000247",91);
		chrom2int.put("un_gl000248",92);
		chrom2int.put("un_gl000249",93);
		}
	
	/** open BDB env */
	@Override
	public void open() throws DatabaseException, IOException
		{
		super.open();
		System.err.println("Opening BDB....");
		DatabaseConfig dbConfig=new DatabaseConfig();
		dbConfig.setAllowCreate(true);
		dbConfig.setReadOnly(false);
		dbConfig.setTransactional(false);
		this.rsToInfo= new DefaultPrimaryDB<Integer, RsInfo>(
				getEnvironment(),
				null,
				"rs2info",
				dbConfig,
				new IntegerBinding(),
				new RsInfoBinding()
				);
		
		dbConfig=new DatabaseConfig();
		dbConfig.setAllowCreate(true);
		dbConfig.setReadOnly(false);
		dbConfig.setTransactional(false);
		currentRsList=new SetOfIntegersDB(getEnvironment(), null, "currentRsList", dbConfig);
		System.err.println("EndOpen BDB");
		}
	
	@Override
	public void close()
		{
		System.err.println("Closing");
		if(rsToInfo!=null) rsToInfo.safeClose();
		rsToInfo=null;
		if(currentRsList!=null) currentRsList.safeClose();
		currentRsList=null;
		 System.err.println("Super.close()");
		super.close();
		System.err.println("End Closing");
		}
	
	@Override
	protected int processArg(String[] args, int optind)
		{
		int n= super.processArg(args, optind);
		if(n!=-1) return n;
		if(args[optind].equals("-p"))
			{
			this.program =args[++optind];
			return optind;
			}
		else if(args[optind].equals("-bowtie-home"))
			{
			this.bowtieHome =new File(args[++optind]);
			return optind;
			}
		
		else if(args[optind].equals("-flanking-size"))
			{
			this.flankingLength =new Integer(args[++optind]);
			return optind;
			}
		else if(args[optind].equals("-bowtie-db"))
			{
			this.bowtieDBDir = new File(args[++optind]);
			return optind;
			}
		else if(args[optind].equals("-blat-binary"))
			{
			this.blatPath = new File(args[++optind]);
			return optind;
			}
		else if(args[optind].equals("-uniq"))
			{
			this.keep_only_uniq_match=true;
			return optind;
			}
		else if(args[optind].equals("-fasta-dir"))
			{
			this.fastaDir=new File(args[++optind]);
			return optind;
			}
		return -1;
		}
	@Override
	public File getDefaultHome()
		{
		return new File("/operon/lindenb/build37bdb");//TODO
		}
	@Override
	protected void usage(PrintStream out)
		{
		super.usage(out);
		out.println("Options");
		out.println(" -uniq keep only uniq match");
		out.println(" -blat-binary <path> default:"+ blatPath);
		out.println(" -bowtie-home <path> default:"+bowtieHome);
		out.println(" -fasta-dir <dir> default:"+fastaDir);
		out.println(" -bowtie-db-directory <name> default:"+bowtieDBDir);
		out.println(" -flanking-size <int> default:"+flankingLength);
		out.println(" -p <program>");
		out.println("    load-dbsnp");
		out.println("         <files|stdin>");
		out.println("    map-flanking");
		out.println("    blat-flanking");
		out.println("    dump");
		out.println("    json");
		}
	
	
	
	private void run(String args[])
		{
		int optind = processArgs(args);
		try
			{
			if(this.program==null)
				{
				System.err.println("undefined program.");
				return;
				}
			else if(program.equals("dump"))
				{
				exportAllResults();
				}
			else if(program.equals("json"))
				{
				exportAsJson();
				}
			else if(program.equals("load-dbsnp"))
				{
				readDbSnpXML(null,optind,args);
				}
			else if(program.equals("map-flanking"))
				{
				/** list all bowtie files */
				this.all_bowtite_files.clear();
				
		        for(File f: this.bowtieDBDir.listFiles(new FilenameFilter()
			     	{
			     	@Override
			     	public boolean accept(File dir, String name)
			     		{
			     		return name.endsWith(".ebwt");
			     		}
			     	}))
		        	{
		        	int i=f.getName().indexOf('.');
		        	this.all_bowtite_files.add(f.getName().substring(0,i));
		        	}
				
				if(optind!=args.length)
					{
					System.err.println("Too Many arguments");
					return;
					}
				open();
				
				MappingTask task=new MappingFlankBowtie();
				System.err.println("Starting mapping loop");
				mappingLoop(task);
				close();
				}
			else if(program.equals("blat-flanking"))
				{
				if(optind!=args.length)
					{
					System.err.println("Too Many arguments");
					return;
					}
				open();
				MappingTask task=new MappingFlankBlat();
				mappingLoop(task);
				close();
				}
			else
				{
				System.err.println("Unknown program");
				return;
				}
			} 
		catch(Exception err)
			{
			err.printStackTrace();
			}
		System.err.println("End.");
		}
	
	private void readDbSnpXML(Transaction txn,int optind,String args[]) throws Exception,SAXException
		{
		if(optind==args.length)
			{
			System.err.println("Files missing");
			return;
			}
		
		open();
		PrimaryDB<String, String> rs2snp = openRsToSnp(txn);
		while(optind<args.length)
			{
			String fname=args[optind++];
			InputStream in=openStream(fname);
			LOG.info("Start reading XML "+fname);
			SAXParserFactory factory= SAXParserFactory.newInstance();
			factory.setNamespaceAware(true);
			SAXParser parser;
			try
				{
				parser=factory.newSAXParser();
				parser.parse(in, new DBSnpHandler(txn));
				LOG.info("Count(SNP) "+this.rsToInfo.count());
				}
			catch(Exception err)
				{
				throw new SAXException(err);
				}
			in.close();
			LOG.info("End reading XML");
			}
		rs2snp.close();
		close();	
		}
	
	private int chromName(String s)
		{
		s=s.trim().toLowerCase();
		if(s.startsWith("chrom")) s=s.substring(5);
		else if(s.startsWith("chr")) s=s.substring(3);
		else if(s.startsWith("k")) s=s.substring(1);
		if(s.equals("x")) return 23;
		if(s.equals("y")) return 24;
		if(s.equals("m")) return 25;
		if( (s.length()==1 && Character.isDigit(s.charAt(0))) ||
			(s.length()==2 && Character.isDigit(s.charAt(0)) && Character.isDigit(s.charAt(1)))
			)
			{
			return Integer.parseInt(s);
			}
		
		Integer i=chrom2int.get(s);
		if(i==null) throw new IllegalArgumentException("chromName="+s);
		return i;
		}
	
	private String int2chrom(int i)
		{
		if(i>0 && i<23) return String.valueOf(i);
		if(i==23) return "X";
		if(i==24) return "Y";
		if(i==25) return "M";
		
		for(String s:chrom2int.keySet())
			{
			if(chrom2int.get(s).equals(i)) return s;
			}
		 throw new IllegalArgumentException("chromID="+i);
		}
	

	private PrimaryDB<String, String> openRsToSnp(Transaction txn) throws DatabaseException
		{
		DatabaseConfig dbConfig=new DatabaseConfig();
		dbConfig.setAllowCreate(true);
		dbConfig.setReadOnly(false);
		dbConfig.setTransactional(false);
		return new DefaultPrimaryDB<String, String>(getEnvironment(),txn,"rs2snp",dbConfig,new StringBinding(),new StringBinding());
		}
	
	/**
	 * Map the flanking ends using bowtie
	 *
	 */
	class MappingFlankBowtie implements MappingTask
		{
		@Override
		public void init(Transaction txn) throws Exception
			{
			}
		@Override
		public void dispose() throws Exception
			{
			
			}
		
		@Override
		public int getPoolSize()
			{
			return 10000000;//1E7
			}
		
		public void map(Transaction txn) throws Exception
			{
			assert(Build37Mapper.this.flankingLength>0);
			LOG.info("mapping "+currentRsList.count()+" snps");
			if(currentRsList.count()==0L) return;
			
			File tmpFile=File.createTempFile("_bowtie_", ".fa");
			PrintWriter out= new PrintWriter(tmpFile);
			
			Walker<Integer,Byte> c= Build37Mapper.this.currentRsList.openWalker(txn);
			while(c.getNext()==OperationStatus.SUCCESS)
				{
				Integer name= c.getKey();
				RsInfo info= Build37Mapper.this.rsToInfo.get(txn, name);
				
				if(info==null) throw new RuntimeException("info is null");
				
				//LOG.info(info.name+" "+seq5);
				out.println(">L_"+info.rsId);
				out.println(info.getSeq5(Build37Mapper.this.flankingLength));
				out.println(">R_"+info.rsId);
				out.println(info.getSeq3(Build37Mapper.this.flankingLength));
				
				
				}
			c.close();
			
			out.flush();
			out.close();
			
			
			
			DatabaseConfig cfg= new DatabaseConfig();
			cfg.setAllowCreate(true);
			cfg.setReadOnly(false);
			BowtieHit.HitDB hitsDB = new BowtieHit.HitDB(getEnvironment(),txn,"hits",cfg);
			
			
			
			//parse the results
			parseBowtieResults(txn,tmpFile,hitsDB);
			
			
			c= Build37Mapper.this.currentRsList.openWalker(txn);
			while(c.getNext()==OperationStatus.SUCCESS)
				{
				Integer name= c.getKey();
				RsInfo info= Build37Mapper.this.rsToInfo.get(txn, name);
				List<BowtieHit.CandidatePair> candidates= new ArrayList<CandidatePair>();
				
				List<BowtieHit> left= hitsDB.get(txn, "L_"+info.rsId);
				List<BowtieHit> right= hitsDB.get(txn, "R_"+info.rsId);
				for(int i=0;i< left.size();++i)
					{
					BowtieHit L= left.get(i);
					for(int j=0;j< right.size();++j)
						{
						final int max_len_deletion=10;
						BowtieHit R= right.get(j);
						if(R.orientation!=L.orientation) continue;
						if(R.chromId!=L.chromId) continue;
						if(R.orientation=='+')
							{
							if(R.position < L.position+Build37Mapper.this.flankingLength) continue;
							if(R.position - (L.position+Build37Mapper.this.flankingLength) > max_len_deletion) continue;
							BowtieHit.CandidatePair candidate= new BowtieHit.CandidatePair(L,R,Build37Mapper.this.flankingLength);
							if(!candidates.isEmpty())
								{
								int cmp= candidates.get(0).compareTo( candidate) ;
								if(cmp > 0)
									{
									candidates.clear();
									}
								else if(cmp<0)
									{
									continue;
									}
								}
							candidates.add(candidate);
							Collections.sort(candidates);
							}
						else if(R.orientation=='-')
							{
							if(L.position < R.position+Build37Mapper.this.flankingLength) continue;
							if(L.position - (R.position+Build37Mapper.this.flankingLength) > max_len_deletion) continue;
							BowtieHit.CandidatePair candidate= new BowtieHit.CandidatePair(L,R,Build37Mapper.this.flankingLength);
							if(!candidates.isEmpty())
								{
								int cmp= candidates.get(0).compareTo( candidate) ;
								if(cmp > 0)
									{
									candidates.clear();
									}
								else if(cmp<0)
									{
									continue;
									}
								}
							candidates.add(candidate);
							Collections.sort(candidates);
							}
						}
					
					}
				
				
			
				if(candidates.size()!=1 &&
						Build37Mapper.this.keep_only_uniq_match
					)
					{
					LOG.info("multiple hits for "+name);
					continue;
					}
				info.leftHitCount= left.size();
				info.rightHitCount= right.size();
				
				for(int i=0;i< candidates.size();++i)
					{
					BowtieHit.CandidatePair candidate= candidates.get(i);
					
					
					RsInfo.Match seq= new RsInfo.Match();
					seq.chromId=candidate.getChromId();
					seq.strand=candidate.getStrand();
					seq.start=candidate.getSNPStart();
					seq.end=candidate.getSNPEnd();
					
					seq.comment="Mapped with Bowtie. (flanking:"+flankingLength+"pb)";
					//seq.buildId=buildDest;
					seq.bound_size= flankingLength;
					
					info.positions37.add(seq);
					}
				 Build37Mapper.this.rsToInfo.put(txn, name,info);
				
				 //apres, il faut sauver leftHitCount
				 if(candidates.isEmpty())
						{
						LOG.info("No candidate for "+name);
						}
				 
				}
			c.close();
			
			
			hitsDB.clear(txn);
			hitsDB.close();
			tmpFile.delete();
			LOG.info("end mapping ");
			}
	}
	
	/** map the flanking ends using BLAT */
	class MappingFlankBlat implements MappingTask
		{
		
		@Override
		public void init(Transaction txn) throws Exception
			{
			}
		@Override
		public void dispose() throws Exception
			{
			
			}
		
		
		@Override
		public int getPoolSize()
			{
			return 10000;//1E4
			}
		
		public void map(Transaction txn) throws Exception
			{
			assert(Build37Mapper.this.flankingLength>0);
			LOG.info("mapping "+currentRsList.count()+" snps");
			if(currentRsList.count()==0L) return;
			
			File tmpFile=File.createTempFile("_blat_", ".fa");
			PrintWriter out= new PrintWriter(tmpFile);
			DatabaseEntry key= new DatabaseEntry();
			//DatabaseEntry data= new DatabaseEntry();
			Walker<Integer,Byte> c= Build37Mapper.this.currentRsList.openWalker(txn);
			while(c.getNext()==OperationStatus.SUCCESS)
				{
				Integer name= c.getKey();
				RsInfo info= Build37Mapper.this.rsToInfo.get(txn, name);
				
				if(info==null) throw new RuntimeException("info is null");
				
				
				
				
					//LOG.info(info.name+" "+seq5);
					out.println(">L_"+info.rsId);
					out.println(info.getSeq5(Build37Mapper.this.flankingLength));
					out.println(">R_"+info.rsId);
					out.println(info.getSeq3(Build37Mapper.this.flankingLength));
				
				}
			c.close();
			
			out.flush();
			out.close();
			
			
			
			DatabaseConfig cfg= new DatabaseConfig();
			cfg.setAllowCreate(true);
			cfg.setReadOnly(false);
			BlatHit.HitDB hitsDB = new BlatHit.HitDB(getEnvironment(),txn,cfg);
			
			
			//parse the results
			callBlat(txn,tmpFile,hitsDB);
			
			
			c=Build37Mapper.this.currentRsList.openWalker(txn);
			while(c.getNext()==OperationStatus.SUCCESS)
				{
				Integer name= Build37Mapper.this.currentRsList.getKeyBinding().entryToObject(key);
				RsInfo info= Build37Mapper.this.rsToInfo.get(txn, name);
				List<BlatHit.CandidatePair> candidates= new ArrayList<BlatHit.CandidatePair>();
				
				List<BlatHit> left= hitsDB.get(txn, "L_"+info.rsId);
				List<BlatHit> right= hitsDB.get(txn, "R_"+info.rsId);
				//System.err.println(name);
				//System.err.println("L:"+left);
				//System.err.println("R:"+right);
				for(int i=0;i< left.size();++i)
					{
					BlatHit L= left.get(i);
					for(int j=0;j< right.size();++j)
						{
						final int max_len_deletion=10;
						BlatHit R= right.get(j);
						if(R.strand!=L.strand) continue;
						if(R.t_chromId!=L.t_chromId) continue;

						if(R.strand=='+')
							{
							if(R.t_start < L.t_end)
								{
								continue;
								}
							if(R.t_start - L.t_end > max_len_deletion)
								{
								continue;
								}
							
							BlatHit.CandidatePair candidate= new BlatHit.CandidatePair(L,R);
							if(!candidates.isEmpty())
								{
								int compare= candidates.get(0).compareTo(candidate);
								if(compare>0)
									{
									candidates.clear();
									}
								else if(compare<0)
									{
									continue;
									}
								}
							candidates.add(candidate);
							Collections.sort(candidates);
							}
						else if(R.strand=='-')
							{
							if(R.t_end > L.t_start)
								{
								continue;
								}
							if(L.t_start - R.t_end > max_len_deletion)
								{
								continue;
								}
						
							BlatHit.CandidatePair candidate= new BlatHit.CandidatePair(L,R);
							if(!candidates.isEmpty())
								{
								int compare= candidates.get(0).compareTo(candidate);
								if(compare>0)
									{
									candidates.clear();
									}
								else if(compare<0)
									{
									continue;
									}
								}
							candidates.add(candidate);
							Collections.sort(candidates);
							}
						}
					
					}
				
				
				
				if(candidates.size()!=1 &&
						Build37Mapper.this.keep_only_uniq_match
					)
					{
					LOG.info("multiple hits for "+name);
					continue;
					}
				
				info.leftHitCount= left.size();
				info.rightHitCount= right.size();
				//save each position
				for(int i=0;i< candidates.size();++i)
					{
					BlatHit.CandidatePair candidate= candidates.get(i);
					
					
					RsInfo.Match seq= new RsInfo.Match();
					seq.chromId=candidate.getChromId();
					seq.strand=candidate.getStrand();
					seq.start=candidate.getSNPStart();
					seq.end=candidate.getSNPEnd();
					
					seq.comment="Mapped with BLAT. Right:"+candidate.R+" Left:"+candidate.L;
					//seq.buildId= buildDest;
					seq.bound_size=flankingLength;
					
					
					info.positions37.add(seq);
					}
				 Build37Mapper.this.rsToInfo.put(txn, name,info);
				 
				 //apres, il faut sauver leftHitCount
				 if(candidates.isEmpty())
						{
						LOG.info("No candidate for "+name);
						}
				}
			c.close();
			
			hitsDB.clear(txn);
			hitsDB.close();
			tmpFile.delete();
			LOG.info("end mapping ");
			}
		}
	
	
	private void mappingLoop(MappingTask task) throws Exception
		{
		long now=System.currentTimeMillis();
		long nSkipped=0L;
		LOG.warning("Entering LOOP "+task.getClass());
		Transaction txn=null;
		int count=0;
		assert(task.getPoolSize()>0);

		Walker<Integer,RsInfo> c=null;
		task.init(txn);
		try
			{
			this.currentRsList.clear(txn);
			c=this.rsToInfo.openWalker(txn);
			OperationStatus status;
			while(true)
				{
				status=c.getNext();
				
				if(count==task.getPoolSize() ||
					status!=OperationStatus.SUCCESS)
					{
					if(count>0)
						{
						assert(this.currentRsList.count()>0);
						LOG.info("Calling task");
						task.map(txn);
						count=0;
						this.currentRsList.clear(txn);
						}
					if(status!=OperationStatus.SUCCESS)
						{
						break;
						}
					}
				
				Integer name= c.getKey();
				RsInfo info= c.getValue();
				assert(name.equals(info.rsId));
				assert(name>0);
				if(info.isMapped())
					{
					++nSkipped;
					LOG.info(name+" already mapped");
					if(nSkipped%100000==0)
						{
						System.err.println("Already Mapped "+name+" "+info);
						}
					continue;
					}
				
				if(!currentRsList.add(txn,name))
					{
					System.err.println("Cannot insert "+name);
					}
				else
					{
					++count;
					if(count==1) System.err.println("Starting new list with rs"+name);
					if(System.currentTimeMillis()-now > 60*1000)
						{
						System.err.println("Added rs"+name +" to list count="+count);
						now=System.currentTimeMillis();
						}
					}
				}
			LOG.warning("Exiting LOOP");
			}
		catch(Exception err)
			{
			throw err;
			}
		finally
			{
			task.dispose();
			if(c!=null) c.close();
			}
		System.err.println("Loop done.");
		}
	
	
	
	private void callBlat(Transaction txn,File readFile,BlatHit.HitDB database)
	throws DatabaseException,IOException,InterruptedException
		{
		 final File outFile=new File("/tmp/_blat.out");
		 Runtime runtime=Runtime.getRuntime();
		 //File chromDir=new File("/home/pierre/tmp/BOWTIE");//TODO
		
		 for(File fastaFile: this.fastaDir.listFiles(new FilenameFilter()
	     	{
	     	@Override
	     	public boolean accept(File dir, String name)
	     		{
	     		return name.endsWith(".nib");
	     		}
	     	}))
	    	 {
	    	 String cmdarray[]={
	    			this.blatPath.toString(),
	    			"-t=dna","-q=dna","-noHead",
	    			fastaFile.toString(),
	                readFile.toString(),
	                outFile.toString()
	                };
	        
		 	LOG.info("sending blat to "+fastaFile);
		 	Process proc=runtime.exec(cmdarray);
		 	int returnValue= proc.waitFor();
	        if(returnValue!=0)
	                {
	                System.err.print("Arguments :\n");
	                for(String s:cmdarray) System.err.print(s+" ");
	                System.err.println();
	                ///throw new IOException("blat returned value!=0 : "+returnValue);
	                }
	        LOG.info("reading result "+fastaFile);
	        
	        BufferedReader in=new BufferedReader(new FileReader(outFile));
	        String line;
	        String token[];
	     
	        while((line=in.readLine())!=null)
	            {
	            if(line.trim().length()==0) continue;
	            if(!Character.isDigit(line.charAt(0))) continue;
	            token= TAB.split(line);
	            token[13]=String.valueOf(chromName(token[13]));
	            BlatHit hit= new BlatHit(token);
	            database.put(txn, hit.q_name, hit);
	            }
	        
	    	}
	    
		}
	
	
	private void parseBowtieResults(
			Transaction txn,
			File readFile,
			BowtieHit.HitDB database
			) throws DatabaseException,IOException,InterruptedException
        {
        //final File outFile=File.createTempFile("_bowtie", ".out");
        final File outFile=new File("/tmp/_bowtie.out");
        Runtime runtime=Runtime.getRuntime();
        
        
        
        for(String bowtieFile: this.all_bowtite_files)
	    	 {
		     String cmdarray[];
		cmdarray=new String[]{
		                this.bowtieHome.toString()+"/bowtie",
		                "-f",
		                //"-a",//report all alignments
		                "-v","3",
		                //"--un","/tmp/_unaligned.txt",//write unalignedreads/pairs to file(s)"
		                new File(this.bowtieDBDir,bowtieFile).toString(),
		                readFile.toString(),
		                outFile.toString()
		                };
			if(keep_only_uniq_match)
				{
				cmdarray=new String[]{this.bowtieHome.toString()+"/bowtie","-f",
					new File(this.bowtieDBDir,bowtieFile).toString(),
	                                readFile.toString(),
        	                        outFile.toString()
                	                };
				}		        


		        LOG.info("sending bowtie ");
		        Process proc=runtime.exec(cmdarray);
		        if(proc.waitFor()!=0)
		                {
		                for(String s:cmdarray) System.err.print(s+" ");
		                throw new IOException("bowtie returned value!=0");
		                }
		        LOG.info("reading result ");
		        //read results
		        BufferedReader in=new BufferedReader(new FileReader(outFile));
		        String line;
		        String token[];
		     
		        while((line=in.readLine())!=null)
		            {
		            token= TAB.split(line);
		            BowtieHit hit= new BowtieHit();
		            assert(token.length>=7);
		            assert(token[1].equals("+") || token[1].equals("-"));
		            hit.readName= token[0];
		            //RsInfo info = this.snpToInfo.get(txn,  hit.readName);
		            hit.orientation=(byte)token[1].charAt(0);
		            hit.chromId=chromName(token[2]);
		            hit.position=Integer.parseInt(token[3]);
		            if(token.length>7 && token[7].length()>0)
		                {
		                for(String mism: COMMA.split(token[7]))
		                    {
		                    int dot= mism.indexOf(':');
		                    assert(dot!=-1);
		                    int lt= mism.indexOf('>',dot+1);
		                    assert(lt>dot);
		                    assert(lt+2==mism.length());
		                    BowtieHit.Mismatch m= new  BowtieHit.Mismatch();
		                    m.position= Integer.parseInt(mism.substring(0,dot));
		                    //if( m.position == Math.min(info.seq5.length(),boundSize)) continue;//ignore the SNP
		                    m.query=(byte)(mism.charAt(dot+1));
		                    m.object=(byte)(mism.charAt(lt+1));
		                    hit.mismatches.add(m);
		                    }
		               }
		         if(database.put(txn, hit.readName, hit)!=OperationStatus.SUCCESS)
		        	 {
		        	 LOG.info("Cannot put "+hit);
		        	 }
		         }
		    in.close();
	    	}
    LOG.info("end bowtie n="+database.count());
    }

	
/** export all results */
private void exportAllResults()
	throws Exception
	{
	open();
	Transaction txn=null;

	
	XMLOutputFactory factory= XMLOutputFactory.newInstance();
	//factory.setProperty(XMLOutputFactory., value);
	XMLStreamWriter w=factory.createXMLStreamWriter(System.out);
	
	Walker<Integer,RsInfo> c= this.rsToInfo.openWalker(txn);
	w.writeStartDocument("UTF-8", "1.0");
	w.writeStartElement("re-mapping");
	
	while(c.getNext()==OperationStatus.SUCCESS)
		{
		
		Integer name= c.getKey();
		RsInfo snpInfo= c.getValue();
		
		w.writeCharacters("\n");
		
		w.writeStartElement("snp");
		w.writeAttribute("name","rs"+name);
		
			
	
		
		
		w.writeStartElement("ncbi");
		
			w.writeCharacters("\n");
			w.writeStartElement("seq5");
			w.writeCharacters(snpInfo.seq5);
			w.writeEndElement();
			w.writeCharacters("\n");
			
			w.writeStartElement("obs");
			w.writeCharacters(snpInfo.observed);
			w.writeEndElement();
			w.writeCharacters("\n");
			
			w.writeStartElement("seq3");
			w.writeCharacters(snpInfo.seq3);
			w.writeEndElement();
			w.writeCharacters("\n");
			
			if(!snpInfo.ncbiHandles.isEmpty())
				{
				w.writeStartElement("handles");
				for(String s:snpInfo.ncbiHandles)
					{
					w.writeStartElement("handle");
					w.writeCharacters(s);
					w.writeEndElement();
					}
				w.writeEndElement();
				}
			
			
		w.writeEndElement();
			
		
		
		w.writeStartElement("dest-mapping");
		w.writeAttribute("count",String.valueOf(snpInfo.positions37.size()));
		w.writeAttribute("leftHitCount",String.valueOf(snpInfo.leftHitCount));
		w.writeAttribute("rightHitCount",String.valueOf(snpInfo.rightHitCount));
		for(RsInfo.Match seq:snpInfo.positions37)
			{
			w.writeStartElement("mapping");
			
			w.writeCharacters("\n");
			
			w.writeStartElement("chrom");
			w.writeCharacters(String.valueOf(seq.chromId));
			w.writeEndElement();
			
			w.writeCharacters("\n");
			
			w.writeStartElement("strand");
			w.writeCharacters(String.valueOf((char)seq.strand));
			w.writeEndElement();
			
			w.writeCharacters("\n");
			
			w.writeStartElement("start");
			w.writeCharacters(String.valueOf(seq.start));
			w.writeEndElement();
			
			w.writeCharacters("\n");
			
			w.writeStartElement("end");
			w.writeCharacters(String.valueOf(seq.end));
			w.writeEndElement();
			
			w.writeCharacters("\n");
			
			
			w.writeEndElement();
			}
		
		w.writeEndElement();
		
		w.writeEndElement();
		w.writeCharacters("\n");
		}
	w.writeEndElement();
	w.writeEndDocument();
	w.flush();
	w.close();
	c.close();

	close();
	}

/** export as JSON */
private void exportAsJson()
	throws Exception
	{
	open();
	Transaction txn=null;

	
	PrintStream w= System.out;
	
	Walker<Integer,RsInfo> c= this.rsToInfo.openWalker(txn);
	
	
	while(c.getNext()==OperationStatus.SUCCESS)
		{
		
		Integer name= c.getKey();
		RsInfo snpInfo= c.getValue();
		
		
		w.print("#1\trs"+name+"\t");
		w.print(""+snpInfo.ncbiHandles.size()+"\t");
			
		w.print("{");
		w.print("\"seq5\":\""+snpInfo.seq5+"\",");
		w.print("\"obs\":\""+snpInfo.observed+"\",");
		w.print("\"seq3\":\""+snpInfo.seq3+"\",");
		w.print("\"map-count\":"+snpInfo.positions37.size()+",");
		w.print("\"left-hit-count\":"+snpInfo.leftHitCount+",");
		w.print("\"right-hit-count\":"+snpInfo.rightHitCount+",");
		w.print("\"handles\":[");
		boolean b=false;
		for(String s:snpInfo.ncbiHandles)
			{
			if(b) w.print(",");
			b=true;
			w.print("\""+s+"\"");
			}
		w.print("]");
		w.print("}");
		w.println();
	
			
		
			
			
		
			
		
		
	
		for(RsInfo.Match seq:snpInfo.positions37)
			{
			w.print("#2\trs"+name+"\t");
			
			w.print("chr"+int2chrom(seq.chromId));
			w.print("\t");
			
			
			w.print(String.valueOf((char)seq.strand));
			w.print("\t");
			
			w.print(String.valueOf(seq.start));
			w.print("\t");
			
			w.print(String.valueOf(seq.end));
			w.println();
			}
		
		if(snpInfo.positions37.isEmpty())
			{
			w.println("#3\trs"+name);
			}
		}
	
	w.flush();
	c.close();
	close();
	}


public static void main(String[] args)
	{
	try
		{
		Build37Mapper app= new Build37Mapper();
		app.run(args);
		System.err.println("Done.");
		}
	catch (Exception err)
		{
		err.printStackTrace();
		}
	}
}
